package com.spider.moudle.crawler.forum;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.LoginDownloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.browser.CommonDriverUtil;
import com.spider.moudle.entity.*;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.ErrorLogUtil;
import com.spider.moudle.util.SettingsUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;

import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;

/**
 * @author zouyang
 * @date 2017/12/19 15:07
 * @description TODO
 */
public class ZhiHuCrawler extends AbstractCrawler {
    private static ArrayList<Cookie> cookies = new ArrayList<>();
    private static Setting siteEntity;
    private static boolean useProxy = true;
    private static int bound = 10;
    private static Random random = new Random();
    private static Downloader authDownloader = DownloaderFactory.buildDownloader();
    private static LoginDownloader loginDownloader = null;
    private static int againGet = 3;

    static {
        siteEntity = SettingsUtil.getSettingBySite("ZhiHu");
        try {
            authDownloader.addHeader("authorization", "oauth c3cef7c66a1843f8b3a9e6a1e3160e20");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    @Override
    public void generateTask() {
        List<Integer> topicIdList = getTopicList();
        if (topicIdList != null && topicIdList.size() > 0) {
            BlockingQueue<String> itemUrlQueue = new LinkedBlockingQueue<>(1000);
            ExecutorService es = ExecutorServiceFactoy.buildExecutorService(topicIdList.size(), ExecutorServiceFactoy.getName(siteEntity.getName(), siteEntity.getIndex()), new NamedThreadFactory(siteEntity.getName(), "获取知乎子话题线程"));
            //根据各个专题ID,获取子话题
            for (int i = 0; i < topicIdList.size(); i++) {
                int topicId = topicIdList.get(i);
                es.submit(new CreateChildTopicThread(topicId, itemUrlQueue));
            }
            es.shutdown();
            ExecutorService taskThreaService = ExecutorServiceFactoy.buildExecutorService(siteEntity.getTask_thread_count(), ExecutorServiceFactoy.getName(siteEntity.getName(), siteEntity.getIndex()), new NamedThreadFactory(siteEntity.getName(), "生成知乎问题任务线程"));
            while (!(itemUrlQueue.isEmpty() && es.isTerminated())) {
                if (!itemUrlQueue.isEmpty()) {
                    taskThreaService.submit(new CreateTaskThread(itemUrlQueue.poll()));
                }
            }
            taskThreaService.shutdown();
        }
    }

    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        if (TaskType.SPEECH == task.getTaskType()) {
            SpeechBean speechBean = new SpeechBean();
            ResponseEntity entity = null;
            try {
                //如果url 包含answer，这是打开的页面第一个的回答者就是问题提出者，否则，就只有通过查看问题日志拿到问题提出者
                if (task.getUrl().contains("answer")) {
                    boolean flag = false;
                    int getNum = 0;
                    while (getNum < againGet) {
                        try {
                            //请求失败则重新请求，最多重新请求5次
                            Thread.sleep(random.nextInt(bound) * 1000);
                            entity = downloader.get(task.getUrl(), useProxy);
                            if (entity != null && entity.getStatus_code() == 200) {
                                String userLink = entity.getDocument().select("div.ContentItem.AnswerItem").get(0).select("meta[itemprop=\"url\"]").attr("content");
                                if (userLink != null && !userLink.equals("")) {
                                    flag = true;
                                    break;
                                }
                            }
                            getNum++;
                            continue;
                        } catch (Exception ex) {
                            getNum++;
                            continue;
                        }
                    }
                    if (flag) {
                        Document document = entity.getDocument();
                        String userLink = document.select("div.ContentItem.AnswerItem").get(0).select("meta[itemprop=\"url\"]").attr("content");
                        speechBean.setPublish_user_id(userLink.substring(userLink.indexOf("/people/") + 8));
                        speechBean.setPublish_user_jsonid(siteEntity.getName() + "_" + speechBean.getPublish_user_id());
                        speechBean.setPublish_user(document.select("div.ContentItem.AnswerItem").get(0).select("meta[itemprop=\"name\"]").attr("content"));
                        String src = document.select("div.ContentItem.AnswerItem").get(0).select("meta[itemprop='image']").attr("content");
                        speechBean.setPublish_user_photo(src.substring(0, src.lastIndexOf("_")) + src.substring(src.lastIndexOf(".")));
                        speechBean.setTitle(document.select("h1.QuestionHeader-title").text());
                        speechBean.setReply_count(Integer.valueOf(document.select("meta[itemprop='answerCount']").attr("content")));
                        speechBean.setContent(CommonUtil.handle(document.select("div.QuestionHeader-detail span.RichText[itemprop='text']").text()));
                        speechBean.setUrl(task.getUrl());
                        speechBean.setSpider_time(new Date());
                        speechBean.setPublish_time(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(document.select("div.QuestionPage>meta[itemprop='dateCreated']").attr("content").replaceAll("[a-zA-Z]", " ").trim()));
                        Elements imgElements = document.select("div.QuestionRichText.QuestionRichText--expandable.QuestionRichText--collapsed img");
                        if (imgElements != null && imgElements.size() > 0) {
                            Set<String> imgSet = new HashSet<>();
                            for (Element e : imgElements) {
                                String imgSrc = e.attr("src");
                                if (imgSrc != null && !"".equals(imgSrc) && imgSrc.startsWith("http")) {
                                    imgSet.add(imgSrc);
                                }
                            }
                            List<String> imgList = new ArrayList<>();
                            imgList.addAll(imgSet);
                            speechBean.setImg_urls(imgList);
                        }
                        speechBean.setRead_count(Integer.valueOf(document.select("div.NumberBoard.QuestionFollowStatus-counts.NumberBoard--divider strong").get(1).text().replaceAll(",", "")));
                        if (speechBean.getReply_count() > 0) {
                            List<SpeechBean> replys = getReplySpeechBean(speechBean.getUrl().substring(0, speechBean.getUrl().indexOf("/answer")), speechBean.getReply_count());
                            speechBean.setReply_nodes(replys);
                        }
                        JSONObject dataJsonObj = new JSONObject();
                        dataJsonObj.put("data", speechBean.toJSONObject());
                        return dataJsonObj;
                    }
                } else {
                    //查看所有日志，知乎会重新生成_xsrf，并且在请求头里加上X-Xsrftoken，作为二次验证，目前还没有想到办法处理，故现在遇到这种情况作为匿名提问处理，
                    boolean flag = false;
                    int getNum = 0;
                    while (getNum < againGet) {
                        try {
                            //请求失败则重新请求，最多重新请求5次
                            Thread.sleep(random.nextInt(bound) * 1000);
                            entity = downloader.get(task.getUrl(), useProxy);
                            if (entity != null && entity.getStatus_code() == 200) {

                                String str = entity.getDocument().select("h1.QuestionHeader-title").text();
                                if (str != null && !str.equals("")) {
                                    flag = true;
                                    break;
                                }
                            }
                            getNum++;
                            continue;
                        } catch (Exception ex) {
                            getNum++;
                            continue;
                        }
                    }
                    if (flag) {
                        Document document = entity.getDocument();
                        speechBean.setPublish_user_id("");
                        speechBean.setPublish_user_jsonid("");
                        speechBean.setPublish_user("匿名");
                        speechBean.setPublish_user_photo("");
                        speechBean.setTitle(document.select("h1.QuestionHeader-title").text());
                        speechBean.setReply_count(Integer.valueOf(document.select("meta[itemprop='answerCount']").attr("content")));
                        speechBean.setContent(CommonUtil.handle(document.select("div.QuestionHeader-detail span.RichText[itemprop='text']").text()));
                        speechBean.setUrl(task.getUrl());
                        speechBean.setSpider_time(new Date());
                        speechBean.setPublish_time(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(document.select("div.QuestionPage>meta[itemprop='dateCreated']").attr("content").replaceAll("[a-zA-Z]", " ").trim()));
                        Elements imgElements = document.select("div.QuestionRichText.QuestionRichText--expandable.QuestionRichText--collapsed img");
                        if (imgElements != null && imgElements.size() > 0) {
                            Set<String> imgSet = new HashSet<>();
                            for (Element e : imgElements) {
                                String imgSrc = e.attr("src");
                                if (imgSrc != null && !"".equals(imgSrc) && imgSrc.startsWith("http")) {
                                    imgSet.add(imgSrc);
                                }
                            }
                            List<String> imgList = new ArrayList<>();
                            imgList.addAll(imgSet);
                            speechBean.setImg_urls(imgList);
                        }
                        speechBean.setRead_count(Integer.valueOf(document.select("div.NumberBoard.QuestionFollowStatus-counts.NumberBoard--divider strong").get(1).text().replaceAll(",", "")));
                        if (speechBean.getReply_count() > 0) {
                            List<SpeechBean> replys = getReplySpeechBean(speechBean.getUrl(), speechBean.getReply_count());
                            speechBean.setReply_nodes(replys);
                        }
                    }
                    JSONObject dataJsonObj = new JSONObject();
                    dataJsonObj.put("data", speechBean.toJSONObject());
                    return dataJsonObj;
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        } else if (TaskType.USER == task.getTaskType()) {
            ResponseEntity entity = null;
            try {
                UserBean userBean = getUserBean(task.getUrl());
                if (userBean != null) {
                    if (userBean.getFans_count() > 0) {
                        int n = 0;
                        String followerUrl = "";
                        ResponseEntity followerEntity = null;
                        List<UserBean> followerUsers = Collections.synchronizedList(new ArrayList<>());
                        int offset = 0;
                        while (n < siteEntity.getMax_page_count()) {
                            followerUrl = "https://www.zhihu.com/api/v4/members/" + userBean.getPublish_user_id() + "/followers?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset=" + offset + "&limit=20";
                            boolean flag = false;
                            int getNum = 0;
                            while (getNum < againGet) {
                                try {
                                    //请求失败则重新请求，最多重新请求5次
                                    Thread.sleep(random.nextInt(bound) * 1000);
                                    followerEntity = authDownloader.get(followerUrl, useProxy);
                                    if (followerEntity != null && followerEntity.getStatus_code() == 200) {
                                        if (followerEntity.getJSONObject().getJSONArray("data") != null) {
                                            flag = true;
                                            break;
                                        }
                                    }
                                    getNum++;
                                    continue;
                                } catch (Exception ex) {
                                    getNum++;
                                    continue;
                                }
                            }

                            if (flag) {
                                JSONArray data = followerEntity.getJSONObject().getJSONArray("data");
                                ExecutorService followUserEs = ExecutorServiceFactoy.buildExecutorService(5, ExecutorServiceFactoy.getName(siteEntity.getName(), siteEntity.getIndex()), new NamedThreadFactory(siteEntity.getName(), "知乎抓取粉丝用户信息的线程"));
                                for (int i = 0; i < data.size(); i++) {
                                    String userUrl = "https://www.zhihu.com/people/" + data.getJSONObject(i).getString("url_token");
                                    followUserEs.submit(new GetUserBeanThread(userUrl, followerUsers));
                                }
                                followUserEs.shutdown();
                                while (true) {
                                    if (followUserEs.isTerminated()) {
                                        break;
                                    }
                                    Thread.sleep(2000);
                                }
                                if (n != 0 && data.size() < 20) {
                                    break;
                                }
                            }
                            offset = offset + 20;
                            n++;
                        }
                        userBean.setFans_details(followerUsers);
                    }
                    if (userBean.getFollow_count() > 0) {
                        int n = 0;
                        String followUrl = "";
                        ResponseEntity followEntity = null;
                        List<UserBean> followUsers = Collections.synchronizedList(new ArrayList<>());
                        int offset = 0;
                        while (n < siteEntity.getMax_page_count()) {
                            followUrl = "https://www.zhihu.com/api/v4/members/" + userBean.getPublish_user_id() + "/followees?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset=" + offset + "&limit=20";
                            boolean flag = false;
                            int getNum = 0;
                            while (getNum < againGet) {
                                try {
                                    //请求失败则重新请求，最多重新请求5次
                                    Thread.sleep(random.nextInt(bound) * 1000);
                                    followEntity = authDownloader.get(followUrl, useProxy);
                                    if (followEntity != null && followEntity.getStatus_code() == 200) {
                                        if (followEntity.getJSONObject().getJSONArray("data") != null) {
                                            flag = true;
                                            break;
                                        }
                                    }
                                    getNum++;
                                    continue;
                                } catch (Exception ex) {
                                    getNum++;
                                    continue;
                                }
                            }

                            if (flag) {
                                JSONArray data = followEntity.getJSONObject().getJSONArray("data");
                                ExecutorService fes = ExecutorServiceFactoy.buildExecutorService(5, ExecutorServiceFactoy.getName(siteEntity.getName(), siteEntity.getIndex()), new NamedThreadFactory(siteEntity.getName(), "知乎抓取关注者的线程"));
                                for (int i = 0; i < data.size(); i++) {
                                    String userUrl = "https://www.zhihu.com/people/" + data.getJSONObject(i).getString("url_token");
                                    fes.submit(new GetUserBeanThread(userUrl, followUsers));
                                }
                                fes.shutdown();
                                while (true) {
                                    if (fes.isTerminated()) {
                                        break;
                                    }
                                    Thread.sleep(2000);
                                }
                                if (n != 0 && data.size() < 20) {
                                    break;
                                }
                            }
                            offset = offset + 20;
                            n++;
                        }
                        userBean.setFollow_details(followUsers);
                    }
                    JSONObject userJsonObj = new JSONObject();
                    userJsonObj.put("data", userBean.toJSONObject());
                    return userJsonObj;
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
        return null;
    }

    public List<SpeechBean> getReplySpeechBean(String questionUrl, int replyCount) {
        ResponseEntity entity = null;
        try {
            List<SpeechBean> replySpeechBeans = Collections.synchronizedList(new ArrayList<>());
            boolean flag = false;
            int getNum = 0;
            while (getNum < againGet) {
                try {
                    //请求失败则重新请求，最多重新请求5次
                    Thread.sleep(random.nextInt(bound) * 1000);
                    entity = downloader.get(questionUrl, useProxy);
                    if (entity != null && entity.getStatus_code() == 200) {
                        if (entity.getDocument().select("h1.QuestionHeader-title").text() != null && !"".equals(entity.getDocument().select("h1.QuestionHeader-title").text())) {
                            flag = true;
                            break;
                        }
                    }
                    getNum++;
                    continue;
                } catch (Exception ex) {
                    getNum++;
                    continue;
                }
            }
            int offset = 0;
            if (flag) {
                Document document = entity.getDocument();
                String questionTitle = document.select("h1.QuestionHeader-title").text();
                String questionId = questionUrl.substring(questionUrl.indexOf("question/") + 9);
                Elements elements = document.select("div.Card>div.List div.List-item");
                offset = elements.size();
                for (Element element : elements) {
                    SpeechBean speechBean = new SpeechBean();
                    String userLink = element.select("div.AuthorInfo.AnswerItem-authorInfo>meta[itemprop='url']").attr("content");
                    speechBean.setPublish_user_id(userLink.substring(userLink.indexOf("people/") + 7));
                    speechBean.setPublish_user_jsonid(siteEntity.getName() + "_" + speechBean.getPublish_user_id());
                    speechBean.setPublish_user(element.select("div.AuthorInfo.AnswerItem-authorInfo>meta[itemprop='name']").attr("content"));
                    String src = element.select("div.AuthorInfo.AnswerItem-authorInfo>meta[itemprop=\"image\"]").attr("content");
                    speechBean.setPublish_user_photo(src.substring(0, src.lastIndexOf("_")) + src.substring(src.lastIndexOf(".")));
                    speechBean.setTitle(questionTitle);
                    speechBean.setLike_count(Integer.valueOf(element.select("div.ContentItem>meta[itemprop=\"upvoteCount\"]").attr("content")));
                    speechBean.setReply_count(Integer.valueOf(element.select("div.ContentItem>meta[itemprop='commentCount']").attr("content")));
                    speechBean.setContent(CommonUtil.handle(element.select("div.RichContent-inner").html()));
                    speechBean.setUrl(element.select("div.ContentItem>meta[itemprop='url']").attr("content"));
                    speechBean.setSpider_time(new Date());
                    speechBean.setPublish_time(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(element.select("div.ContentItem>meta[itemprop='dateCreated']").attr("content").replaceAll("[a-zA-Z]", " ").trim()));
                    speechBean.setAncestor_id(questionId);
                    speechBean.setParent_id(questionId);
                    speechBean.setId(speechBean.getUrl().substring(speechBean.getUrl().indexOf("answer/") + 7));
                    Elements imgElements = element.select("img");
                    if (imgElements != null && imgElements.size() > 0) {
                        Set<String> imgSet = new HashSet<>();
                        for (Element e : imgElements) {
                            String imgSrc = e.attr("src");
                            if (imgSrc != null && !"".equals(imgSrc) && imgSrc.startsWith("http")) {
                                imgSet.add(imgSrc);
                            }
                        }
                        List<String> imgList = new ArrayList<>();
                        imgList.addAll(imgSet);
                        speechBean.setImg_urls(imgList);
                    }
                    if (speechBean.getReply_count() > 0) {
                        //抓问题的评论
                        List<SpeechBean> comments = getComments(speechBean.getId(), questionTitle, questionId, speechBean.getReply_count());
                        if (comments != null && comments.size() > 0) {
                            speechBean.setReply_nodes(comments);
                        }
                    }
                    replySpeechBeans.add(speechBean);
                }
                //继续抓剩下的回答
                int n = 0;
                ExecutorService es = ExecutorServiceFactoy.buildExecutorService(siteEntity.getReply_thread_count(), ExecutorServiceFactoy.getName(siteEntity.getName(), siteEntity.getIndex()), new NamedThreadFactory(siteEntity.getName(), "抓取知乎回复线程"));
                while (n < siteEntity.getMax_page_count()) {
                    es.submit(new GetReplysThread(questionTitle, questionId, questionUrl, offset, replySpeechBeans));
                    if (offset > replyCount) {
                        break;
                    }
                    offset = offset + 20;
                    n++;
                }
                es.shutdown();
                while (true) {//所有的抓回复的线程执行结束后才继续运行主线程
                    if (es.isTerminated()) {
                        break;
                    }
                    Thread.sleep(2000);
                }
                return replySpeechBeans;
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            log.error(ex.getMessage(), ex);
        }
        return null;
    }

    private class GetReplysThread implements Runnable {
        String questionUrl;
        String questionId;
        String questionTitle;
        int offset;
        List<SpeechBean> speechBeanList;

        public GetReplysThread(String questionTitle, String questionId, String questionUrl, int offset, List<SpeechBean> speechBeans) {
            this.questionTitle = questionTitle;
            this.questionId = questionId;
            this.questionUrl = questionUrl;
            this.offset = offset;
            speechBeanList = speechBeans;
        }

        @Override
        public void run() {
            String nextAnswerUrl = "https://www.zhihu.com/api/v4/questions" + questionUrl.substring(questionUrl.indexOf("/question") + 9) + "/answers";
            nextAnswerUrl += "?sort_by=default&include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Crelevant_info%2Cquestion%2Cexcerpt%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics";
            nextAnswerUrl += "&limit=20&offset=" + offset;

            boolean nextFlag = false;
            int num = 0;
            ResponseEntity nextEntity = null;
            try {
                while (num < againGet) {
                    try {
                        //请求失败则重新请求，最多重新请求5次
                        Thread.sleep(random.nextInt(bound) * 1000);
                        nextEntity = authDownloader.get(nextAnswerUrl, useProxy);
                        if (nextEntity != null && nextEntity.getStatus_code() == 200) {
                            if (nextEntity.getJSONObject().getJSONArray("data") != null) {
                                nextFlag = true;
                                break;
                            }
                        }
                        num++;
                        continue;
                    } catch (Exception ex) {
                        num++;
                        continue;
                    }
                }
                if (nextFlag) {
                    JSONObject entityJsonObject = nextEntity.getJSONObject();
                    JSONArray dataArr = entityJsonObject.getJSONArray("data");
                    for (int i = 0; i < dataArr.size(); i++) {
                        JSONObject data = dataArr.getJSONObject(i);
                        SpeechBean speechBean = new SpeechBean();
                        speechBean.setId(data.getString("id"));
                        Element contentElement = new Element("div");
                        contentElement.html(data.getString("content"));
                        Elements imgElements = contentElement.select("img");
                        if (imgElements != null && imgElements.size() > 0) {
                            Set<String> imgSet = new HashSet<>();
                            for (Element e : imgElements) {
                                String imgSrc = e.attr("src");
                                if (imgSrc != null && !"".equals(imgSrc) && imgSrc.startsWith("http")) {
                                    imgSet.add(imgSrc);
                                }
                            }
                            List<String> imgList = new ArrayList<>();
                            imgList.addAll(imgSet);
                            speechBean.setImg_urls(imgList);
                        }
                        speechBean.setParent_id(questionId);
                        speechBean.setAncestor_id(questionId);
                        speechBean.setLike_count(data.getInteger("voteup_count"));
                        JSONObject author = data.getJSONObject("author");
                        speechBean.setPublish_user(author.getString("name"));
                        speechBean.setPublish_user_id(author.getString("url_token"));
                        speechBean.setPublish_user_jsonid(siteEntity.getName() + "_" + speechBean.getPublish_user_id());
                        speechBean.setPublish_user_photo(author.getString("avatar_url").substring(0, author.getString("avatar_url").lastIndexOf("_")) + author.getString("avatar_url").substring(author.getString("avatar_url").lastIndexOf(".")));
                        speechBean.setPublish_time(new Date(data.getLong("created_time") * 1000 - 288 * 100000));
                        speechBean.setSpider_time(new Date());
                        speechBean.setUrl(questionUrl + "/answer/" + speechBean.getId());
                        speechBean.setContent(CommonUtil.handle(data.getString("content")));
                        speechBean.setReply_count(data.getInteger("comment_count"));
                        speechBean.setTitle(questionTitle);
                        if (speechBean.getPublish_user_id() != null && !"".equals(speechBean.getPublish_user_id())) {
                            createUserTask(speechBean.getPublish_user_id(), siteEntity.getSite() + "_" + speechBean.getPublish_user_id());
                        }
                        if (speechBean.getReply_count() > 0) {
                            List<SpeechBean> comments = getComments(speechBean.getId(), questionTitle, questionId, speechBean.getReply_count());
                            if (comments != null && comments.size() > 0) {
                                speechBean.setReply_nodes(comments);
                            }
                        }
                        speechBeanList.add(speechBean);
                    }
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
    }

    /**
     * @author zouyang
     * @date 2018/1/2 20:12
     * @description 抓取评论
     * @params
     */
    private List<SpeechBean> getComments(String answerId, String questionTitle, String questionId, int replyCounts) {
        String commentUrl = "https://www.zhihu.com/api/v4/answers/" + answerId + "/comments";
        try {
            List<SpeechBean> commentList = Collections.synchronizedList(new ArrayList<>());
            int n = 0;
            int offset = 0;
            ExecutorService commentEs = ExecutorServiceFactoy.buildExecutorService(siteEntity.getReply_thread_count(), ExecutorServiceFactoy.getName(siteEntity.getName(), siteEntity.getIndex()), new NamedThreadFactory(siteEntity.getName(), "抓取评论的线程"));
            while (n < siteEntity.getMax_page_count()) {
                commentEs.submit(new CommentsThreat(answerId, questionTitle, questionId, commentUrl, offset, commentList));
                if (offset > replyCounts) {
                    break;
                }
                offset = offset + 20;
                n++;
            }
            commentEs.shutdown();
            while (true) {
                if (commentEs.isTerminated()) {
                    break;
                }
                Thread.sleep(2000);
            }
            return commentList;
        } catch (Exception ex) {
            ex.printStackTrace();
            log.error(ex.getMessage(), ex);
        }
        return null;
    }

    private class CommentsThreat implements Runnable {
        private String answerId;
        private String questionTitle;
        private String questionId;
        private String commentUrl;
        private int offset;
        private List<SpeechBean> commentList;

        public CommentsThreat(String answerId, String questionTitle, String questionId, String commentUrl, int offset, List<SpeechBean> commentList) {
            this.answerId = answerId;
            this.questionId = questionId;
            this.questionTitle = questionTitle;
            this.commentUrl = commentUrl;
            this.offset = offset;
            this.commentList = commentList;
        }

        @Override
        public void run() {
            ResponseEntity commentEntity = null;
            try {
                String nextCommentUrl = commentUrl + "?include=data%5B*%5D.author%2Ccollapsed%2Creply_to_author%2Cdisliked%2Ccontent%2Cvoting%2Cvote_count%2Cis_parent_author%2Cis_author&order=normal&limit=20&offset=" + offset + "&status=open";
                boolean flag = false;
                int getNum = 0;
                while (getNum < againGet) {
                    try {
                        //请求失败则重新请求，最多重新请求5次
                        Thread.sleep(random.nextInt(bound) * 1000);
                        commentEntity = authDownloader.get(nextCommentUrl, useProxy);
                        if (commentEntity != null && commentEntity.getStatus_code() == 200) {
                            if (commentEntity.getJSONObject().getJSONArray("data") != null) {
                                flag = true;
                                break;
                            }
                        }
                        getNum++;
                        continue;
                    } catch (Exception ex) {
                        getNum++;
                        continue;
                    }
                }
                if (flag) {
                    JSONObject commentJsonObject = commentEntity.getJSONObject();
                    JSONArray dataArr = commentJsonObject.getJSONArray("data");
                    for (int i = 0; i < dataArr.size(); i++) {
                        JSONObject data = dataArr.getJSONObject(i);
                        SpeechBean speechBean = new SpeechBean();
                        JSONObject author = data.getJSONObject("author").getJSONObject("member");
                        speechBean.setPublish_user_id(author.getString("url_token"));
                        speechBean.setPublish_user(author.getString("name"));
                        speechBean.setTitle(questionTitle);
                        speechBean.setContent(CommonUtil.handle(data.getString("content")));
                        speechBean.setUrl(data.getString("url"));
                        speechBean.setSpider_time(new Date());
                        speechBean.setPublish_time(new Date(data.getLong("created_time") * 1000 - 288 * 100000));
                        speechBean.setLike_count(data.getInteger("vote_count"));
                        speechBean.setAncestor_id(questionId);
                        speechBean.setParent_id(answerId);
                        speechBean.setId(data.getString("id"));
                        speechBean.setPublish_user_jsonid(siteEntity.getName() + speechBean.getPublish_user_id());
                        speechBean.setPublish_user_photo(author.getString("avatar_url").substring(0, author.getString("avatar_url").lastIndexOf("_")) + author.getString("avatar_url").substring(author.getString("avatar_url").lastIndexOf(".")));
                        speechBean.setTitle(questionTitle);
                        speechBean.setLike_count(data.getInteger("vote_count"));
                        if (speechBean.getPublish_user_id() != null && !"".equals(speechBean.getPublish_user_id())) {
                            createUserTask(speechBean.getPublish_user_id(), siteEntity.getSite() + "_" + speechBean.getPublish_user_id());
                        }
                        if (data.getJSONObject("reply_to_author") != null) {
                            List<SpeechBean> conversationList = getConversationSpeech(speechBean.getId(), questionTitle, questionId);
                            if (conversationList != null && conversationList.size() > 0) {
                                speechBean.setReply_nodes(conversationList);
                            }
                        }
                        commentList.add(speechBean);
                    }
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }

        /**
         * @author zouyang
         * @date 2018/1/2 20:12
         * @description 抓取对话
         * @params
         */
        private List<SpeechBean> getConversationSpeech(String conversationId, String questionTitle, String questionId) {
            String conversationUrl = "https://www.zhihu.com/api/v4/comments/" + conversationId + "/conversation?include=%24%5B*%5D.author%2Creply_to_author%2Ccontent%2Cvote_count";
            List<SpeechBean> conversationList = new ArrayList<>();
            ResponseEntity conversationEntity = null;
            try {

                boolean flag = false;
                int getNum = 0;
                while (getNum < againGet) {
                    try {
                        //请求失败则重新请求，最多重新请求5次
                        Thread.sleep(random.nextInt(bound) * 1000);
                        conversationEntity = authDownloader.get(conversationUrl, useProxy);
                        if (conversationEntity != null && conversationEntity.getStatus_code() == 200 && JSON.parseArray(conversationEntity.getResponse_body()) != null) {
                            if (conversationEntity.getResponse_body() != null) {
                                flag = true;
                                break;
                            }
                        }
                        getNum++;
                        continue;
                    } catch (Exception ex) {
                        getNum++;
                        continue;
                    }
                }
                if (flag) {
                    JSONArray dataArray = JSON.parseArray(conversationEntity.getResponse_body());
                    for (int i = 0; i < dataArray.size(); i++) {
                        JSONObject data = dataArray.getJSONObject(i);
                        if (data.getBoolean("is_delete")) {
                            continue;
                        }
                        SpeechBean speechBean = new SpeechBean();
                        JSONObject author = data.getJSONObject("author").getJSONObject("member");
                        speechBean.setPublish_user_id(author.getString("url_token"));
                        speechBean.setPublish_user(author.getString("name"));
                        speechBean.setTitle(questionTitle);
                        speechBean.setContent(CommonUtil.handle(data.getString("content")));
                        speechBean.setUrl(data.getString("url"));
                        speechBean.setSpider_time(new Date());
                        speechBean.setPublish_time(new Date(data.getLong("created_time") * 1000 - 288 * 100000));
                        speechBean.setLike_count(data.getInteger("vote_count"));
                        speechBean.setAncestor_id(questionId);
                        speechBean.setParent_id(conversationId);
                        speechBean.setId(data.getString("id"));
                        speechBean.setPublish_user_jsonid(siteEntity.getName() + speechBean.getPublish_user_id());
                        speechBean.setPublish_user_photo(author.getString("avatar_url").substring(0, author.getString("avatar_url").lastIndexOf("_")) + author.getString("avatar_url").substring(author.getString("avatar_url").lastIndexOf(".")));
                        speechBean.setTitle(questionTitle);
                        speechBean.setLike_count(data.getInteger("vote_count"));
                        if (speechBean.getPublish_user_id() != null && !"".equals(speechBean.getPublish_user_id())) {
                            createUserTask(speechBean.getPublish_user_id(), siteEntity.getSite() + "_" + speechBean.getPublish_user_id());
                        }
                        conversationList.add(speechBean);
                    }
                    return conversationList;
                }
            } catch (Exception ex) {
                ex.printStackTrace();
                log.error(ex.getMessage(), ex);
            }
            return null;
        }
    }

    public UserBean getUserBean(String userUrl) {
        ResponseEntity entity = null;
        try {

            boolean flag = false;
            int getNum = 0;
            while (getNum < againGet) {
                try {
                    //请求失败则重新请求，最多重新请求5次
                    Thread.sleep(random.nextInt(bound) * 1000);
                    entity = authDownloader.get(userUrl, useProxy);
                    if (entity != null && entity.getStatus_code() == 200) {
                        if (entity.getDocument() != null && JSON.parseObject(entity.getDocument().getElementById("data").attr("data-state")).getJSONObject("entities").getJSONObject("users") != null) {
                            flag = true;
                            break;
                        }
                    }
                    getNum++;
                    continue;
                } catch (Exception ex) {
                    getNum++;
                    continue;
                }
            }
            if (flag) {
                UserBean userBean = new UserBean();
                Document document = entity.getDocument();
                userBean.setUrl(userUrl);
                userBean.setPublish_user_id(userBean.getUrl().substring(userBean.getUrl().indexOf("people/") + 7));
                JSONObject userData = JSON.parseObject(document.getElementById("data").attr("data-state")).getJSONObject("entities").getJSONObject("users").getJSONObject(userBean.getPublish_user_id());
                if (userData != null && userData.size() > 0) {
                    userBean.setPublish_user(userData.getString("name"));
                    String imgUrl = userData.getString("avatarUrl");
                    userBean.setPublish_user_photo(imgUrl.substring(0, imgUrl.lastIndexOf("_")) + imgUrl.substring(imgUrl.lastIndexOf(".")));
                    userBean.setArticle_count(Integer.valueOf(userData.getString("questionCount")));
                    userBean.setFans_count(Integer.valueOf(userData.getString("followerCount")));
                    userBean.setFollow_count(Integer.valueOf(userData.getString("followingCount")));
                    userBean.setJson_id(siteEntity.getName() + "_" + userBean.getPublish_user_id());
                    userBean.setReply_article_count(Integer.valueOf(userData.getString("answerCount")));
                    userBean.setIntroduction(userData.getString("description"));
                    return userBean;
                }
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            log.error(ex.getMessage(), ex);
        }
        return null;
    }

    public class GetUserBeanThread implements Runnable {
        private String userUrl;
        private List<UserBean> userBeans;

        public GetUserBeanThread(String userUrl, List<UserBean> userBeans) {
            this.userUrl = userUrl;
            this.userBeans = userBeans;
        }

        @Override
        public void run() {
            ResponseEntity entity = null;
            try {
                boolean flag = false;
                int getNum = 0;
                while (getNum < againGet) {
                    try {
                        //请求失败则重新请求，最多重新请求5次
                        Thread.sleep(random.nextInt(bound) * 1000);
                        entity = authDownloader.get(userUrl, useProxy);
                        if (entity != null && entity.getStatus_code() == 200) {
                            if (entity.getDocument() != null && JSON.parseObject(entity.getDocument().getElementById("data").attr("data-state")).getJSONObject("entities").getJSONObject("users") != null) {
                                flag = true;
                                break;
                            }
                        }
                        getNum++;
                        continue;
                    } catch (Exception ex) {
                        getNum++;
                        continue;
                    }
                }
                if (flag) {
                    UserBean userBean = new UserBean();
                    Document document = entity.getDocument();
                    userBean.setUrl(userUrl);
                    userBean.setPublish_user_id(userBean.getUrl().substring(userBean.getUrl().indexOf("people/") + 7));
                    JSONObject userData = JSON.parseObject(document.getElementById("data").attr("data-state")).getJSONObject("entities").getJSONObject("users").getJSONObject(userBean.getPublish_user_id());
                    if (userData != null && userData.size() > 0) {
                        userBean.setPublish_user(userData.getString("name"));
                        String imgUrl = userData.getString("avatarUrl");
                        userBean.setPublish_user_photo(imgUrl.substring(0, imgUrl.lastIndexOf("_")) + imgUrl.substring(imgUrl.lastIndexOf(".")));
                        userBean.setArticle_count(Integer.valueOf(userData.getString("questionCount")));
                        userBean.setFans_count(Integer.valueOf(userData.getString("followerCount")));
                        userBean.setFollow_count(Integer.valueOf(userData.getString("followingCount")));
                        userBean.setJson_id(siteEntity.getName() + "_" + userBean.getPublish_user_id());
                        userBean.setReply_article_count(Integer.valueOf(userData.getString("answerCount")));
                        userBean.setIntroduction(userData.getString("description"));
                        userBeans.add(userBean);
                    }
                }
            } catch (Exception ex) {
                ex.printStackTrace();
                log.error(ex.getMessage(), ex);
            }
        }
    }

    /**
     * @author zouyang
     * @date 2017/12/25 11:19
     * @description 获取专题Id列表
     * @params
     */
    public List<Integer> getTopicList() {
        ResponseEntity responseEntity = null;
        try {
            //拿到所有专题的id
            List<Integer> dataIdList = new ArrayList<>();
            while (true) {
                boolean flag = false;
                int getNum = 0;
                while (getNum < againGet) {
                    try {
                        //请求失败则重新请求，最多重新请求5次
                        Thread.sleep(random.nextInt(bound) * 1000);
                        responseEntity = downloader.get(siteEntity.getIndex(), useProxy);
                        if (responseEntity != null && responseEntity.getStatus_code() == 200) {
                            if (responseEntity.getDocument() != null) {
                                flag = true;
                                break;
                            }
                        }
                        getNum++;
                        continue;
                    } catch (Exception ex) {
                        getNum++;
                        continue;
                    }
                }
                Document document = null;
                Elements elements = null;
                if (flag) {
                    document = responseEntity.getDocument();
                    elements = document.select("li.zm-topic-cat-item");

                    for (int i = 0; i < elements.size(); i++) {
                        Element element = elements.get(i);
                        dataIdList.add(Integer.valueOf(element.attr("data-id")));
                    }
                    if (dataIdList.size() > 0) {
                        return dataIdList;
                    }
                }
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            ErrorLogUtil.error(ex);
        }
        return null;
    }

    /**
     * 将话题url添加到队列中
     */
    private class CreateChildTopicThread implements Runnable {
        private int topicId;
        private BlockingQueue itemUrlQueue;

        public CreateChildTopicThread(int topicId, BlockingQueue<String> itemUrlQueue) {
            this.topicId = topicId;
            this.itemUrlQueue = itemUrlQueue;
        }

        @Override
        public void run() {
            ResponseEntity entity = null;
            Map<String, String> params = new HashMap<>();
            Map<String, Object> map = new HashMap<>();
            int offset = 0;
            //每个专题取5页子话题
            for (int n = 0; n < 5; n++) {
                try {
                    map.put("topic_id", topicId);
                    map.put("offset", offset + 20 * n);
                    map.put("hash_id", "");
                    params.put("method", "next");
                    params.put("params", JSON.toJSONString(map));

                    boolean flag = false;
                    int getNum = 0;
                    while (getNum < againGet) {
                        try {
                            //请求失败则重新请求，最多重新请求5次
                            Thread.sleep(random.nextInt(bound) * 1000);
                            entity = downloader.post("https://www.zhihu.com/node/TopicsPlazzaListV2", params, useProxy);
                            if (entity != null && entity.getStatus_code() == 200) {
                                if (entity.getJSONObject().getJSONArray("msg") != null) {
                                    flag = true;
                                    break;
                                }
                            }
                            getNum++;
                            continue;
                        } catch (Exception ex) {
                            getNum++;
                            continue;
                        }
                    }
                    if (flag) {
                        JSONArray jsonArray = entity.getJSONObject().getJSONArray("msg");
                        if (jsonArray != null && !jsonArray.isEmpty()) {
                            for (int m = 0; m < jsonArray.size(); m++) {
                                Element element = new Element("div");
                                element.html(jsonArray.get(m).toString());
                                String itemUrl = "https://www.zhihu.com" + element.select("div.blk>a[target='_blank']").attr("href");
                                itemUrlQueue.put(itemUrl);
                            }
                        } else if (jsonArray == null || jsonArray.size() < 20) {
                            //如果获取到的子话题少于20，说明已将所有的子话题都获取到了，后面的请求将不必继续
                            break;
                        }
                    } else {
                        break;
                    }
                } catch (Exception ex) {
                    ex.printStackTrace();
                    log.error(ex);
                }
            }
        }


    }

    /**
     * 根据话题URL生成任务线程
     */
    private class CreateTaskThread implements Runnable {
        private String itemUrl;

        public CreateTaskThread(String itemUrl) {
            this.itemUrl = itemUrl;
        }

        @Override
        public void run() {
            try {
                speechTask(itemUrl + "/hot");
                speechTask(itemUrl + "/newest");
                speechTask(itemUrl + "/top-answers");
            } catch (Exception ex) {
                ex.printStackTrace();
                log.error(ex.getMessage(), ex);
            }
        }

        public void speechTask(String url) {
            //取问题
            String offset = "";
            for (int j = 0; j < siteEntity.getMax_page_count(); j++) {
                ResponseEntity entity = null;
                Elements elements = null;
                try {
                    if (j == 0) {
                        boolean flag = false;
                        int getNum = 0;
                        while (getNum < againGet) {
                            try {
                                //请求失败则重新请求，最多重新请求5次
                                Thread.sleep(random.nextInt(bound) * 1000);
                                entity = downloader.post(url, useProxy);
                                if (entity != null && entity.getStatus_code() == 200) {
                                    if (entity.getJSONObject() != null && entity.getJSONObject().getJSONArray("msg") != null
                                            && entity.getJSONObject().getJSONArray("msg").get(1) != null) {
                                        flag = true;
                                        break;
                                    }
                                }
                                getNum++;
                                continue;
                            } catch (Exception ex) {
                                getNum++;
                                continue;
                            }
                        }
                        if (flag) {
                            Element responseElement = new Element("div");
                            responseElement.html(entity.getJSONObject().getJSONArray("msg").get(1).toString());
                            elements = responseElement.select("div.feed-item.feed-item-hook[itemprop='question']");
                            if (elements != null && elements.size() > 0) {
                                for (int i = 0; i < elements.size(); i++) {
                                    String questionUrl = "https://www.zhihu.com" + elements.get(i).select("link[itemprop='url']").attr("href");
                                    createSpeechTask(questionUrl);
                                }
                                offset = elements.last().attr("data-score");
                            } else {
                                break;
                            }
                        } else {
                            break;
                        }
                    } else {
                        Map<String, String> nextParamsMap = new HashMap<>();
                        nextParamsMap.put("start", "0");
                        nextParamsMap.put("offset", offset);

                        boolean flag = false;
                        int getNum = 0;
                        while (getNum < againGet) {
                            try {
                                //请求失败则重新请求，最多重新请求5次
                                Thread.sleep(random.nextInt(bound) * 1000);
                                entity = downloader.post(url, nextParamsMap, useProxy);
                                if (entity != null && entity.getStatus_code() == 200) {
                                    if (entity.getJSONObject() != null && entity.getJSONObject().getJSONArray("msg") != null
                                            && entity.getJSONObject().getJSONArray("msg").get(1) != null) {
                                        flag = true;
                                        break;
                                    }
                                }
                                getNum++;
                                continue;
                            } catch (Exception ex) {
                                getNum++;
                                continue;
                            }
                        }
                        if (flag) {
                            Element responseElement = new Element("div");
                            responseElement.html(entity.getJSONObject().getJSONArray("msg").get(1).toString());
                            elements = responseElement.select("div.feed-item.feed-item-hook[itemprop='question']");
                            if (elements != null && elements.size() > 0) {
                                if (elements != null && elements.size() > 0) {
                                    for (int i = 0; i < elements.size(); i++) {
                                        String questionUrl = "https://www.zhihu.com" + elements.get(i).select("link[itemprop='url']").attr("href");
                                        createSpeechTask(questionUrl);
                                    }
                                    offset = elements.last().attr("data-score");
                                }
                            } else {
                                break;
                            }
                        } else {
                            break;
                        }
                    }
                } catch (Exception ex) {
                    ex.printStackTrace();
                    log.error(ex.getMessage(), ex);
                }
            }
        }
    }

    /**
     * @author zouyang
     * @date 2017/12/25 11:48
     * @description 生成任务
     * @params
     */
    public void createSpeechTask(String taskUrl) {
        try {
            String questionId = "";
            if (taskUrl.contains("/answer/")) {
                questionId = taskUrl.substring(taskUrl.indexOf("question/") + 9, taskUrl.indexOf("/answer"));
            } else {
                questionId = taskUrl.substring(taskUrl.indexOf("question/") + 9);
            }
            String taskKey = siteEntity.getSite() + "_" + questionId;
            Task task = new Task(taskUrl, TaskType.SPEECH, WebType.FORLUM, taskKey);
            TaskManager.add(siteEntity.getSpeech_task(), task);
        } catch (Exception e) {
            e.printStackTrace();
            log.error(e.getMessage(), e);
        }

    }

    public void createUserTask(String userId, String taskKey) {
        try {
            String taskUrl = "https://www.zhihu.com/people/" + userId;
            Task task = new Task(taskUrl, TaskType.USER, WebType.FORLUM, taskKey);
            TaskManager.add(siteEntity.getUser_task(), task);
        } catch (Exception e) {
            e.printStackTrace();
            log.error(e.getMessage(), e);
        }
    }

    public static void testLoginByDriver() {
        WebDriver webDriver = null;
        try {
            while (true) {
                if (cookies == null || cookies.isEmpty()) {
                    webDriver = CommonDriverUtil.getDriver();
                    webDriver.get("https://www.zhihu.com/signin");
                    WebElement username = webDriver.findElement(By.name("username"));
                    username.sendKeys("13981707065");
                    WebElement password = webDriver.findElement(By.name("password"));
                    password.sendKeys("aabbcc123");
                    WebElement submit = webDriver.findElement(By.cssSelector("button.Button.SignFlow-submitButton.Button--primary.Button--blue"));
                    submit.click();
                    Thread.sleep(3000);
                    Set<org.openqa.selenium.Cookie> cookieSet = webDriver.manage().getCookies();
                    if (cookieSet == null || cookieSet.isEmpty()) {
                        continue;
                    }
                    String xsrf = "";
                    for (org.openqa.selenium.Cookie cookie : cookieSet) {
                        Cookie c = new Cookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain());
                        cookies.add(c);
                        if ("_xsrf".equals(cookie.getName())) {
                            xsrf = cookie.getValue();
                        }
                    }
                    loginDownloader = DownloaderFactory.buildLoginDownloader(cookies, "UTF-8");
                    loginDownloader.addHeader("X-Xsrftoken", xsrf);
                } else {
                    String xsrf = "";
                    for (Cookie cookie : cookies) {
                        if ("_xsrf".equals(cookie.getName())) {
                            xsrf = cookie.getValue();
                        }
                    }
                    loginDownloader = DownloaderFactory.buildLoginDownloader(cookies, "UTF-8");
                    loginDownloader.addHeader("X-Xsrftoken", xsrf);
                }
                break;
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            ex.printStackTrace();
        }

    }

    public static void main(String[] args) {
        ZhiHuCrawler zhiHuCrawler = new ZhiHuCrawler();
//        zhiHuCrawler.generateTask();
//        Task task = new Task("https://www.zhihu.com/question/264787941/answer/287615186", TaskType.SPEECH, WebType.FORLUM, "ZhiHu_34158677");
        Task task3 = new Task("https://www.zhihu.com/people/gan-jue-hao-duo-liao", TaskType.USER, WebType.FORLUM, "ZhiHu_you-a-you-65");
        try {
//            zhiHuCrawler.crawlItem(task2);
//            JSONObject jsonObject = zhiHuCrawler.crawlItem(task);
            JSONObject jsonObject  = zhiHuCrawler.crawlItem(task3);
            System.out.println(jsonObject);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
